{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## import必要的包"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "d:\\python\\lib\\importlib\\_bootstrap.py:219: RuntimeWarning: numpy.ufunc size changed, may indicate binary incompatibility. Expected 192 from C header, got 216 from PyObject\n",
      "  return f(*args, **kwds)\n",
      "d:\\python\\lib\\importlib\\_bootstrap.py:219: RuntimeWarning: numpy.ufunc size changed, may indicate binary incompatibility. Expected 192 from C header, got 216 from PyObject\n",
      "  return f(*args, **kwds)\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "# 导入评价指标logloss\n",
    "from sklearn.metrics import log_loss\n",
    "\n",
    "from matplotlib import pyplot"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 加载特征工程处理后的数据文件"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pregnants</th>\n",
       "      <th>Plasma_glucose_concentration</th>\n",
       "      <th>blood_pressure</th>\n",
       "      <th>Triceps_skin_fold_thickness</th>\n",
       "      <th>serum_insulin</th>\n",
       "      <th>BMI</th>\n",
       "      <th>Diabetes_pedigree_function</th>\n",
       "      <th>Age</th>\n",
       "      <th>Target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.639947</td>\n",
       "      <td>0.866045</td>\n",
       "      <td>-0.031990</td>\n",
       "      <td>0.670643</td>\n",
       "      <td>-0.181541</td>\n",
       "      <td>0.166619</td>\n",
       "      <td>0.468492</td>\n",
       "      <td>1.425995</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-0.844885</td>\n",
       "      <td>-1.205066</td>\n",
       "      <td>-0.528319</td>\n",
       "      <td>-0.012301</td>\n",
       "      <td>-0.181541</td>\n",
       "      <td>-0.852200</td>\n",
       "      <td>-0.365061</td>\n",
       "      <td>-0.190672</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.233880</td>\n",
       "      <td>2.016662</td>\n",
       "      <td>-0.693761</td>\n",
       "      <td>-0.012301</td>\n",
       "      <td>-0.181541</td>\n",
       "      <td>-1.332500</td>\n",
       "      <td>0.604397</td>\n",
       "      <td>-0.105584</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-0.844885</td>\n",
       "      <td>-1.073567</td>\n",
       "      <td>-0.528319</td>\n",
       "      <td>-0.695245</td>\n",
       "      <td>-0.540642</td>\n",
       "      <td>-0.633881</td>\n",
       "      <td>-0.920763</td>\n",
       "      <td>-1.041549</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-1.141852</td>\n",
       "      <td>0.504422</td>\n",
       "      <td>-2.679076</td>\n",
       "      <td>0.670643</td>\n",
       "      <td>0.316566</td>\n",
       "      <td>1.549303</td>\n",
       "      <td>5.484909</td>\n",
       "      <td>-0.020496</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   pregnants  Plasma_glucose_concentration  blood_pressure  \\\n",
       "0   0.639947                      0.866045       -0.031990   \n",
       "1  -0.844885                     -1.205066       -0.528319   \n",
       "2   1.233880                      2.016662       -0.693761   \n",
       "3  -0.844885                     -1.073567       -0.528319   \n",
       "4  -1.141852                      0.504422       -2.679076   \n",
       "\n",
       "   Triceps_skin_fold_thickness  serum_insulin       BMI  \\\n",
       "0                     0.670643      -0.181541  0.166619   \n",
       "1                    -0.012301      -0.181541 -0.852200   \n",
       "2                    -0.012301      -0.181541 -1.332500   \n",
       "3                    -0.695245      -0.540642 -0.633881   \n",
       "4                     0.670643       0.316566  1.549303   \n",
       "\n",
       "   Diabetes_pedigree_function       Age  Target  \n",
       "0                    0.468492  1.425995       1  \n",
       "1                   -0.365061 -0.190672       0  \n",
       "2                    0.604397 -0.105584       1  \n",
       "3                   -0.920763 -1.041549       0  \n",
       "4                    5.484909 -0.020496       1  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train = pd.read_csv('F:/CSDN/AI/ComputerLearning/201905148140_学习资料/FE_diabetes.csv')\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 768 entries, 0 to 767\n",
      "Data columns (total 9 columns):\n",
      "pregnants                       768 non-null float64\n",
      "Plasma_glucose_concentration    768 non-null float64\n",
      "blood_pressure                  768 non-null float64\n",
      "Triceps_skin_fold_thickness     768 non-null float64\n",
      "serum_insulin                   768 non-null float64\n",
      "BMI                             768 non-null float64\n",
      "Diabetes_pedigree_function      768 non-null float64\n",
      "Age                             768 non-null float64\n",
      "Target                          768 non-null int64\n",
      "dtypes: float64(8), int64(1)\n",
      "memory usage: 54.1 KB\n"
     ]
    }
   ],
   "source": [
    "train.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "768个样本，维度为9"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 分离特征和标签"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train = train['Target']\n",
    "x_train = train.drop(['Target'], axis=1)\n",
    "\n",
    "# 记录特征名称\n",
    "feature_names = x_train.columns"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 模型训练"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 默认参数的Logistic回归"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "# 得到实例\n",
    "lr = LogisticRegression()  # 默认超参数是1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "logloss of each fold is:  [0.48797856 0.53011593 0.4562292  0.422546   0.48392885]\n",
      "cv logloss is: 0.47615970944434044\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "d:\\python\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "d:\\python\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "d:\\python\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "d:\\python\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "d:\\python\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    }
   ],
   "source": [
    "# 交叉验证 评估模型性能并进行参数调优\n",
    "# 分类任务中交叉验证中缺省是采用StratifiedKFold（分层的K折交叉验证）\n",
    "from sklearn.model_selection import cross_val_score\n",
    "loss = cross_val_score(lr, x_train, y_train, cv=5, scoring='neg_log_loss')\n",
    "print ('logloss of each fold is: ', -loss)\n",
    "print ('cv logloss is:', -loss.mean())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 正则化的Logistic Regression及参数调优"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "logistic回归需要调整的超参数：C（正则系数）、L2/L1（正则函数penalty）\n",
    "目标函数：J = C* sum(logloss(f(xi),yi)) + penalty\n",
    "\n",
    "在sklearn框架下：不同学习器的参数调整步骤相同：\n",
    "    设置候选参数集合\n",
    "    生成一个GridSearchCV的实例\n",
    "    调用GridSearchCV的fit函数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 14 candidates, totalling 70 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=4)]: Using backend LokyBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=4)]: Done  10 tasks      | elapsed:    3.3s\n",
      "[Parallel(n_jobs=4)]: Done  48 out of  70 | elapsed:    3.7s remaining:    1.6s\n",
      "[Parallel(n_jobs=4)]: Done  70 out of  70 | elapsed:    3.8s finished\n",
      "d:\\python\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=5, error_score='raise-deprecating',\n",
       "       estimator=LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "          intercept_scaling=1, max_iter=100, multi_class='warn',\n",
       "          n_jobs=None, penalty='l2', random_state=None, solver='warn',\n",
       "          tol=0.0001, verbose=0, warm_start=False),\n",
       "       fit_params=None, iid='warn', n_jobs=4,\n",
       "       param_grid={'penalty': ['l1', 'l2'], 'C': [0.001, 0.01, 0.1, 1, 10, 100, 1000]},\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score='warn',\n",
       "       scoring='neg_log_loss', verbose=5)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "# 设置超参数的搜索范围\n",
    "penaltys = ['l1', 'l2']\n",
    "Cs  = [0.001, 0.01, 0.1, 1, 10, 100, 1000]\n",
    "tuned_parameters = dict(penalty=penaltys, C=Cs)\n",
    "\n",
    "lr_penalty = LogisticRegression()\n",
    "grid = GridSearchCV(lr_penalty, tuned_parameters, cv=5, scoring='neg_log_loss', n_jobs=4, verbose=5) # 4个线程， verbase不等于0，则打印中间结果\n",
    "grid.fit(x_train, y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "上述输出中，refit=True 表示5折交叉验证后，使用得到的最佳超参数结合全体数据再进行模型训练，得到权重系数，作为最终的系数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.4760272455574415\n",
      "{'C': 1, 'penalty': 'l1'}\n"
     ]
    }
   ],
   "source": [
    "# 选择分数最高的打印输出\n",
    "print(-grid.best_score_)\n",
    "print(grid.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>columns</th>\n",
       "      <th>coefficent</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Plasma_glucose_concentration</td>\n",
       "      <td>[1.1284528784451628]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>BMI</td>\n",
       "      <td>[0.6291651780204015]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>pregnants</td>\n",
       "      <td>[0.40962866570996936]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Diabetes_pedigree_function</td>\n",
       "      <td>[0.27933275938588237]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Age</td>\n",
       "      <td>[0.1438515588386752]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Triceps_skin_fold_thickness</td>\n",
       "      <td>[0.02477462045470976]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>serum_insulin</td>\n",
       "      <td>[-0.08405081741024145]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>blood_pressure</td>\n",
       "      <td>[-0.09295913014821269]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                        columns              coefficent\n",
       "1  Plasma_glucose_concentration    [1.1284528784451628]\n",
       "5                           BMI    [0.6291651780204015]\n",
       "0                     pregnants   [0.40962866570996936]\n",
       "6    Diabetes_pedigree_function   [0.27933275938588237]\n",
       "7                           Age    [0.1438515588386752]\n",
       "3   Triceps_skin_fold_thickness   [0.02477462045470976]\n",
       "4                 serum_insulin  [-0.08405081741024145]\n",
       "2                blood_pressure  [-0.09295913014821269]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.DataFrame({'columns':list(feature_names), 'coefficent':list(grid.best_estimator_.coef_.T)}) # coef是权重系数W的属性\n",
    "df.sort_values(by=['coefficent'],ascending=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "L1正则在超参数C取合适值的时候具有稀疏性"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "d:\\python\\lib\\site-packages\\sklearn\\utils\\deprecation.py:125: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "d:\\python\\lib\\site-packages\\sklearn\\utils\\deprecation.py:125: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEKCAYAAADjDHn2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3Wd4VNX69/HvPZNeIEAChF4EKQIBItg44lEUlV4EBClWLGA5IrbHAh5FD3/bUY+NKiAgShFQBKWIgBAQkVAUqZEAMRBKembW82ImEEJC6mRnwv3xmiuz96w989uAc2eXtZYYY1BKKaUuxmZ1AKWUUuWfFgullFIF0mKhlFKqQFoslFJKFUiLhVJKqQJpsVBKKVUgLRZKKaUKpMVCKaVUgbRYKKWUKpCP1QFKS3h4uGnQoIHVMZRSyqts3rz5b2NMREHtPFosRKQr8A5gBz41xkzI9fpbwA3uxSCgujEmzP3aMOB592uvGGOmXeyzGjRoQExMTGnGV0qpCk9EDhSmnceKhYjYgfeBLkAcsElEFhljdmS3McY8nqP9KKCt+3lV4EUgGjDAZve2JzyVVymlVP48ec2iA7DHGLPXGJMBzAZ6XqT9IOBz9/NbgOXGmOPuArEc6OrBrEoppS7Ck8WiNnAox3Kce90FRKQ+0BD4oajbKqWU8jxPXrOQPNblNx76QGCeMcZRlG1F5H7gfoB69eoVJ6NSqoLJzMwkLi6OtLQ0q6OUKwEBAdSpUwdfX99ibe/JYhEH1M2xXAc4nE/bgcDDubbtnGvbVbk3MsZ8DHwMEB0drRNzKKWIi4sjNDSUBg0aIJLX752XHmMMiYmJxMXF0bBhw2K9hydPQ20CmohIQxHxw1UQFuVuJCKXA1WA9TlWLwNuFpEqIlIFuNm9TimlLiotLY1q1appochBRKhWrVqJjrY8dmRhjMkSkUdwfcnbgcnGmFgRGQfEGGOyC8cgYLbJMWWfMea4iIzHVXAAxhljjnsqq1KqYtFCcaGS/pl4tJ+FMWYpsDTXuhdyLb+Uz7aTgckeC+fmyMpi06ejqdf1UWo1uNzTH6eUKocGfOQ6sTHngastTlJ+XfLDfRzeF0uLIwvwnXoLf2zbYHUcpZSXCwkJOfu8a9euhIWF0a1bt3zbT506lcOH87uce3GrVq1i3bp1xdq2qC75YlG3SRuO37EQg1Djy9788uPXVkdSSlUQY8aM4bPPPrtoGy0WXqRBiyuRe5eTZK9GixUj+GnRJKsjKaUqgBtvvJHQ0NB8X583bx4xMTEMHjyYqKgoUlNT2bx5M9dffz3t27fnlltuIT4+HoB3332XFi1a0Lp1awYOHMj+/fv58MMPeeutt4iKiuLHH3/06L5UmIEESyqizmWcGb2Sgx/04OrN/2JF0lFuvOsZvVCmlBd7+etYdhw+VWC7HfGuNtnXLi6mRa1KvNi9ZYmzAfTr14/33nuPiRMnEh0dTWZmJqNGjWLhwoVEREQwZ84cnnvuOSZPnsyECRPYt28f/v7+JCUlERYWxsiRIwkJCeHJJ58slTwXo8Uih5CwCBo8sZyd7/fnpr2vs/z9eDqPfBtfH7vV0ZRSl4Ddu3ezfft2unTpAoDD4SAyMhKA1q1bM3jwYHr16kWvXr3KPJsWi1x8A0Jo8dgifvvoHrocm87KtxKIfngKoUGBVkdTShVRYY8AysvdUMYYWrZsyfr1Fx7hLFmyhDVr1rBo0SLGjx9PbGxsmWbTaxZ5ELsvrR6cRuxl93ND8jdsf6snRxO1m4dSqvSFhoZy+vRpAC6//HISEhLOFovMzExiY2NxOp0cOnSIG264gTfeeIOkpCTOnDlz3raepsUiPyK0HPIffo9+iY4ZGzn6Xlf2HDhodSqllBfp1KkT/fv35/vvv6dOnTosW3bhQBTDhw9n5MiRREVF4XA4mDdvHmPHjqVNmzZERUWxbt06HA4HQ4YMoVWrVrRt25bHH3+csLAwunfvzvz588vkArfk6Djt1aKjo42nJj86uPZzaq54hEPUIKnPHNq3buWRz1FKldzOnTtp3rx5kbYpL6ehPC2vPxsR2WyMiS5oW71mUQj1rhvEsUoR1PzqLoK/7MHKk1O4oVNnq2MppUpJRS8SpUFPQxVS9dY34Rz+DX52aLdiEAsXfkFFOSpTSqmCaLEogtAGUQQ9tJJUv2p03fIgn0//Hw6nFgylVMWnxaKIAsIbUP3RVfwd0pQBe59l1gcvkZrhKHhDpZTyYlosisEWEk7tR5dzOOJa7vr7bRa+/QiJp3VWLqVUxaXForj8gqn74ALi6vdhYMosfnrnLvYfK3hYAaVUOTTldtdD5UuLRUnYfakzfDLxrR+iR9Z37P2gL1v3xludSillobIaovzDDz9k+vTpxcpYHFosSkqEyD6v8Xen8XRmE1nTevLDL7utTqWUKgdKOkS5w5H/9dCRI0cydOjQEuUrCi0WpST8xtGc6fYxbeRPas/vw5crf7Y6klLKYsUZorxBgwaMGzeO6667ji+++IJPPvmEK6+8kjZt2tC3b19SUlIAeOmll5g4cSIAnTt3ZuzYsXTo0IGmTZt6pDe3dsorRZWi7yCtUjj1Ph9MyKpBfHL8Pe7pfSs2mw5zrpQlvnkajvxWcLsj21w/C3PdomYruHVCyXK55R6iPFtAQABr164FIDExkfvuuw+A559/nkmTJjFq1KgL3isrK4uNGzeydOlSXn75ZVasWFEqGbPpkUUpC2j6T3zv/ZZKvtB/2728M3UG6Vl6a61SqvAGDBhw9vn27dvp1KkTrVq1YubMmfmONtunTx8A2rdvz/79+0s9kx5ZeIBP7TaEPPQ9Jz/pwYMHHufdDxK4/75RVA70tTqaUpeWwh4BZB9RjFjiuSxFEBwcfPb58OHDWbBgAW3atGHq1KmsWrUqz238/f0BsNvtZGVllXomPbLwEKnakLBHVpJS5XKeSBzHpHdf4q+kVKtjKaXKmYKGGT99+jSRkZFkZmYyc+bMMkx2Pi0WnhQcTtUHl3Gq1nU8kfoeX//3cWL/SrI6lVKqjBR1iPLU1At/oRw/fjwdO3akS5cuNGvWrCxi50mHKC8LjkxOznmAyr9/yefmZurc+V86XV7T6lRKVUjFGaK8vJ2G8pSSDFGuRxZlwe5L5YGfcqb9QwyS70ieeRdfbfzT6lRKqWwjllT4QlFSWizKis1GSPfXSPvneLraNlJr8WA+XLZFhzlXSnkFLRZlLOAfo8nq9QnRtj+4/qehvDpnJZkOp9WxlFLqorRYWMAn6g7sQ76gsU8iw3bexwuTviI5vfRvdVNKqdKixcIictk/8bt3KdX8nYz561FeeH8Kx3SYc6VUOaXFwkq12hI48nsCQqvyysln+c9/32XPsTNWp1LqkjPi2xGM+HaE1THKNY8WCxHpKiK7RWSPiDydT5s7RGSHiMSKyKwc6x0istX9WOTJnJaq2oigkd9jwpvyWsZrTP/g32zaf9zqVEqpYsoeonzr1q1cffXVtGzZktatWzNnzpw82xd3iHKAVatWsW7dumJnLQqPFQsRsQPvA7cCLYBBItIiV5smwDPAtcaYlsBjOV5ONcZEuR89PJWzXAipTtD935JZ9zrG8QFrJj3Dkl+L949HKVU+BAUFMX36dGJjY/n222957LHHSEq6sFPuJV8sgA7AHmPMXmNMBjAb6JmrzX3A+8aYEwDGmGMezFO++YcSOOxLMpr34V/22SR88RifrtljdSqlVDE1bdqUJk2aAFCrVi2qV69OQkLCeW3yGqJ88+bNXH/99bRv355bbrmF+HjXhGrvvvsuLVq0oHXr1gwcOJD9+/fz4Ycf8tZbbxEVFeWRYclz8uRAgrWBQzmW44COudo0BRCRnwA78JIx5lv3awEiEgNkAROMMQs8mLV88PHDr/8ksr6twfCN/2Px8pO8cmICz3SPwq7DnCtVZK9vfJ1dx3cV2C67TWGuWzSr2oyxHcYWKcfGjRvJyMigcePG563PPUR5ZmYmo0aNYuHChURERDBnzhyee+45Jk+ezIQJE9i3bx/+/v4kJSURFhbGyJEjCQkJ4cknnyxSnuLwZLHI69stdw80H6AJ0BmoA/woIlcYY5KAesaYwyLSCPhBRH4zxpzX7VlE7gfuB6hXr15p57eGzYbPbRNwVqpFtxX/j6oxD/Fk0gReu/M6AnztVqdTShVRfHw8d911F9OmTcNmu/jJnN27d7N9+3a6dOkCuGbKi4yMBKB169YMHjyYXr160atXL4/nzs2TxSIOqJtjuQ6Q+8RcHLDBGJMJ7BOR3biKxyZjzGEAY8xeEVkFtAXOKxbGmI+Bj8E1NpQndsIqtutGQ2h1rlrwMGF/juLhj15l4oibqRLsZ3U0pbxGYY8Aso8opnSdUqqff+rUKW6//XZeeeUVrrrqqgLbG2No2bIl69evv+C1JUuWsGbNGhYtWsT48ePzndfCUzx5zWIT0EREGoqIHzAQyH1X0wLgBgARCcd1WmqviFQREf8c668Fdngwa/nUZiC2O+fQ1DeBlxIe49H3v+DQ8RSrUymlCiEjI4PevXszdOhQ+vfvn2+7nEOUX3755SQkJJwtFpmZmcTGxuJ0Ojl06BA33HADb7zxBklJSZw5c6bA4c1Lk8eKhTEmC3gEWAbsBOYaY2JFZJyIZN/dtAxIFJEdwEpgjDEmEWgOxIjIr+71E4wxl16xAGhyEz53L6ZmgIO3U8by3PvT2Banw5wrVd7NnTuXNWvWMHXqVKKiooiKimLr1q0XtMs5RLnD4WDevHmMHTuWNm3aEBUVxbp163A4HAwZMoRWrVrRtm1bHn/8ccLCwujevTvz588vkwvcOkS5t0j8k8xpvcg6dZTRjicYNHgE/2xWw+pUSpU7xRmi3FOnocobHaL8UlCtMb73rcAnogn/s7/B4s/eYtbPB61OpVSFMKXrlApfKEpKi4U3Ca2B7z3fIPWv4U3f/7F/0atM/HbXuWHOp9x+bhIXpZQqRVosvE1AJex3fYmzRW+e9f2csLUv8eScX8jIchIbf5LY+JNWJ1RKVUBaLLyRjz+2fpMxHR7gXp9v+Efsc9w75SeSjb/VyZRSFZQWC29lsyG3vg43vkhP+zruP/Q0Lyb3429nqNXJlFIVkBYLbyYCnZ6Anh9wrX0nE33+x6vJPUlKybA6mVJe5cBdQzlw11CrY5RrWiwqgraDkUGzudwWx0e+b7JgQ8Fj4SilPKOshij/8MMPmT59eomyFoUnh/tQZanpzcT51KNB1j5ObPgMZ+c3sOngg0pZJnuI8iZNmnD48OGzo8iGhYWd127q1KlcccUV1KpV64L3cDgc2O15jwk3cuRIj+TOjx5ZVCDJEszfhNEl7TvW7vnb6jhKXdKKO0R5gwYNGDduHNdddx1ffPEFn3zyCVdeeSVt2rShb9++pKS4hvx56aWXmDhxIgCdO3dm7NixdOjQgaZNm3qkN7ceWVCBem+KkGYL5gr288rq5fyj6Z1WJ1LKUkdefZX0nQWflk3b5WpTmOsW/s2bUfPZZ4uUo7BDlGcLCAhg7dq1ACQmJnLfffcB8PzzzzNp0iRGjRp1wWdkZWWxceNGli5dyssvv8yKFSuKlLEgemRRgbSMrEyd2nXJtPnT6OA8DielWh1JqUte9hDlU6ZMKXCI8mwDBgw4+3z79u106tSJVq1aMXPmzHxHm+3Tpw8A7du3Z//+/SXOndslf2ThNE6aLPyVy4/aoavVaUqBzYeMpj3ovnMxU9bvYvStba1OpJRlCnsEkH1EUf+z0r1gXNQhyrMFBweffT58+HAWLFhAmzZtmDp1KqtWrcpzG39/Vz8ru91OVlZWiXLn5ZI/sjiYsIdUZzrrGlac202Dr76HUEklKWYuGVlOq+ModUkqzhDleTl9+jSRkZFkZmYyc+ZMT0QtlEu+WNTOCuGOH5003Z/J7uO7rY5TMiOWuB71riK5UmO6ZX7HdzuOWJ1KqUtScYYoT0298NTx+PHj6dixI126dKFZs2ZlET1POkQ58OO1rQlPzOSzJ1vz73tmI+L9t5w6f/ovtuXP82TEh0x8eJDVcZQqM8UZotxTp6HKGx2ivIRmjGnDmWAb7eZu44cD31sdp1TYou7EIT60jJ/PH0fLZiYtpbxV/c+mV/hCUVJaLICPen1GwyeeoeVB+GbGeNId6VZHKrngamQ1vZ3e9rXMXve71WmUUl5Oi4VbtTsG4Khbk65LjzFj21Sr45QK/w4jCJNkzmxdQHJ66d8doVR5VVFOr5emkv6ZaLFwE19f6j/zArWPw57pH5KQklDwRuVdw+tJD6lLL+cKFm4t+tgzSnmjgIAAEhMTtWDkYIwhMTGRgICAYr/HJd/PIqeQGzpja9+aXqu38cHaibx48+tWRyoZmw2/DsO5+ofxTPppHYM61K0QF++Vupg6deoQFxd3wdAal7qAgADq1KlT7O21WOQgItR75gWc/frh+/nXbG93F1eEX2F1rBKRtkNwrnyV6OOL2XLwFtrXr2J1JKU8ytfXl4YNG1odo8LR01C5BF7RkqBut9Ftk+F/y8Z5/6FsaE2cl91Mf/saZq/fY3UapZSX0mKRh1pP/Au7zYc2X23nm33fWB2nxHyuHEE1OUnq9iUcT644PdWVUmVHi0UefGvVInzYcDrFGr5YNIGUzBSrI5XMZTeRGRxJf/meuTGHrE6jlPJCWizyEf7AA5iwSty+NIGp27186HKbHd/ooXSy/8aK9ZtwOr381JpSqsxpsciHPSSEyEcfo+VB+GX+J8Sfibc6Usm0HYIAnc4sY/UfepeIUqpotFhcRFi/ftga1GPQigze3vh/VscpmbB6mEb/ZIDPamat32t1GqWUl9FicRHi60utp54m8rjBseAbthzdYnWkErFFD6MmiTj+WMGh415+HUYpVaa0WBQg5IbO+HeI5o618Nbqf+M0Xjw/RNNbcQSFM9D2A59vPGh1GqWUF9FiUQARIXLs04SmOGmxdCcL9yy0OlLx+fhhbzuYG+2/8P2mbaRnOaxOpJTyEh4tFiLSVUR2i8geEXk6nzZ3iMgOEYkVkVk51g8TkT/cj2GezFmQwJYtqdSjB91i4LMf3uRMxhkr45RMu2HYcXJj2nK+3a4TIymlCsdjxUJE7MD7wK1AC2CQiLTI1aYJ8AxwrTGmJfCYe31V4EWgI9ABeFFELB2novrjj2G3+dD1u0Q+/u1jK6OUTLXGmPrXMcRvNbPW77c4jFLKW3jyyKIDsMcYs9cYkwHMBnrmanMf8L4x5gSAMeaYe/0twHJjzHH3a8uBrh7MWiDfyEjCR9xNp1jDTyumc+iU93Zuk/bDqWWOYj+0ll1HTlkdRynlBTxZLGoDOb9R49zrcmoKNBWRn0Rkg4h0LcK2Za7affciVcIY8n0WEzf9x+o4xde8O86AMAb7rGTmBr3QrZQqmCeLRV5jYefuOuwDNAE6A4OAT0UkrJDbIiL3i0iMiMSUxXDE9pAQaoweTbODDk798D0b4jd4/DM9wjcAW5uB3GKP4YctOzijEyMppQrgyWIRB9TNsVwHyD0DTxyw0BiTaYzZB+zGVTwKsy3GmI+NMdHGmOiIiIhSDZ+fsP798W3UkOGrbfxn/QSynF76RdtuGD4mk66OVcz/5S+r0yilyjlPFotNQBMRaSgifsBAYFGuNguAGwBEJBzXaam9wDLgZhGp4r6wfbN7neXEx4caY8ZQ/e8sGq78g3m/z7M6UvHUaIGpcyXD/Vczc/1+7x+KXSnlUR4rFsaYLOARXF/yO4G5xphYERknIj3czZYBiSKyA1gJjDHGJBpjjgPjcRWcTcA497pyIaRzZ4I6dmTQOhuTN/yXk+knrY5ULNJuGHWdcYQciyHmwAmr4yilyjGpKL9RRkdHm5iYmDL7vNTYWPb368/CqwQevIunO+TZjaR8y0jGTGzKoox2/NBsHO8MbGt1IqVUGRORzcaY6ILaaQ/uYgps2ZLKPXrQbROs2PA5fyb9aXWkovMLRlr15zb5mR9/28PfZ9KtTqSUKqe0WJRAxGOPYrf7MHgN/GfTf7zzvH/7YfiadG7jJ+Zs8t6+I0opz9JiUQK+kZFUGz6Cq7ZncCRmLT/+9aPVkYquVluo2Zp7A9cwa8MBHDoxklIqD1osSqjaffdir1aV+9b48sbG18l0ZFodqejaDaVB1p9UPbWDVbuPFdxeKXXJ0WJRQvaQECJGjaLRvjSqb97PrF2zCt6ovGl9B8YnkBGBq5mx4YDVaZRS5ZAWi1IQ1q8ffo0ace8afz7Z8j8SUxOtjlQ0AZWRlr3pxk9s/P2QToyklLpAkYuFiNhEpJInwngr8fGh+pgnqZKQyrWbknlv63tWRyq69sPwc6bQ3b6BmT/reFFKqfMVqliIyCwRqSQiwcAOYLeIjPFsNO8S0rkzQVddxaB1dr7ZNo9dx3dZHalo6naE8Mu5P/hH5sYcIi1TJ0ZSSp1T2COLFsaYU0AvYClQD7jLY6m8kIhQ46kx+CdnMGCjL69vfN27bqUVgfbDaJS+k4iUPXyzPd7qREqpcqSwxcJXRHxxFYuFxphM8hgF9lIX0KIFlXv04OaNmRzYvYnlB5ZbHaloWg/E2P24P2QtM3TocqVUDoUtFh8B+4FgYI2I1Ad01pw8RDz2KHabnfvWB/Pm5jdJy0qzOlLhBVdDmnfndrOG7QeOsuOw/hUrpVwKVSyMMe8aY2obY24zLgdwjxarzucbGUnVEcOJ2nqKgN/jmL5jutWRiqbdMAKyTtHdN4YZP+tttEopl8Je4H7UfYFbRGSSiGwB/unhbF6r2r33Ya9WjdE/VeLTbZ9wLMWLOro16ARVGvBg6FoW/PIXp9K8sJOhUqrUFfY01N3uC9w3AxHACGCCx1J5OXtIMBGjHiFyzwna7MrgnS3vWB2p8Gw2aDeUxilbqZEZx/wtOjGSUqrwxSJ7mtPbgCnGmF/Je+pT5RbWrx9+jRvzwNpAlvy+kG0J26yOVHhRg0HsjApbx4wNB7zrri6llEcUtlhsFpHvcBWLZSISCjg9F8v7ZXfUCz5ykt7bg3h94+s4jZf8kYXWhMtv5TbHSvYfS+LnfeVm3imllEUKWyzuAZ4GrjTGpAB+uE5FqYsIuf56gq66ij5rHeyJ+5Ule5dYHanw2g0jIOM4PQK26nhRSqlC3w3lBOoAz4vIROAaY4wXnVexRnZHPZ/Tqdz3S1Xe3vw2KZleMu7SZTdCpTo8WOknvt1+hGOnvegWYKVUqSvs3VATgEdxDfWxAxgtIq95MlhFEdCiBZV79uSan05g4o8yafskqyMVjs0ObYfQ+NRGappjzNWJkZS6pBX2NNRtQBdjzGRjzGSgK3C752JVLBGPPYrNZudfm2syLXYaf53xkjuM2g5BgH+Fb2TWzwfJcnjJNRelVKkryqizYTmeVy7tIBWZb82aVB0xnEYb/6LxX07ejHnT6kiFE1YXLruJW7NWcPRkMj/s8qL+IkqpUlXYYvEa8IuITBWRacBm4FXPxap4sjvqPbauCt/tX0bMkRirIxVOu6EEpB6lV8hOZujQ5Updsgp7gftz4CrgK/fjamPMbE8Gq2hcHfVGEbb7MDcfDOP1Ta/jcHrBMOCX3wrB1Xmo0lrW/J7A/r+TrU6klLLARYuFiLTLfgCRQBxwCKjlXqeKIKxfX/waN2bYKuGPhJ0s2LPA6kgFs/tC1J00OvETkbYTzNqoRxdKXYoKOrL4v4s8Jno2WsWT3VHP9/DfjPijNu/+8i6nM05bHatg7YYixsEzkZt1YiSlLlEXLRbGmBsu8tCBBIsh5PrrCbr6Krr8cIL0pON8vO1jqyMVrFpjaNCJm9OXczIlncXbdGIkpS41he1n0SePx40iUt3TASsaV0e9p5BTZ3hqZ2Nm7JzBgVNe0EO6/XACzhyiX5U/tUe3Upegogz38Skw2P34BHgC+ElEdHrVIgpo3pzKPXvS/Pu91Drty8RNXnBGr1k3CKzCQ5V+YuuhJLb/ddLqREqpMlTYYuEEmhtj+hpj+gItgHSgIzDWU+EqsojHHkXsdp7eXJtVcatY99c6qyNdnG8AtBlEg4SV1PJN1qMLpS4xhS0WDYwxR3MsHwOaGmOOAzo7TjFkd9QL/2kX1yZF8MamN8hyZlkd6+LaDUWcmTxXeysLtv7FyVT9q1fqUlHYYvGjiCwWkWEiMgxYhGsu7mAgKb+NRKSriOwWkT0i8nQerw8XkQQR2ep+3JvjNUeO9YuKumPeoNo992KvVo0Hfwziz6Q9zN091+pIF1e9OdTpwE2p35KW6eCrLXFWJ1JKlZHCFouHgSlAFNAWmAY8bIxJNsbkORe3iNiB94FbcZ22GiQiLfJoOscYE+V+fJpjfWqO9T0Ku0PeJLujnt/2Pxmc0IT3t75PUlq+tbd8aD8M/5N/cmeNv/hMJ0ZS6pJR2B7cBlgL/ACsANaYgr8lOgB7jDF7jTEZwGygZ0nCVkRh/frid1ljei47SWraaT749QOrI11cy97gX4kHQteyNyGZ9X8mWp1IKVUGCnvr7B3ARqAfcAfws4j0K2Cz2rh6e2eLc6/Lra+IbBOReSJSN8f6ABGJEZENItKrMDm9kfj4UGPMGIiLZ0xca+bunsueE3usjpU/v2Bo1Y96R76jbmA6M37WC91KXQoKexrqOVyz5A0zxgzFddTw/wrYJq85unMfjXyN6+J5a1xHLNNyvFbPGBMN3Am8LSKNL/gAkfvdBSUmISGhkLtS/gT/4x8EXX0VbRf/QbgjkDc2vVG+T++0G4ZkpfF8vViWxR7l6CmdGEmpiq6wxcJmjMk5PnViIbaNA3IeKdQBDudsYIxJNMakuxc/AdrneO2w++deYBWuayXk2v5jY0y0MSY6IiKikLtS/mR31DOnTvP8rmasj1/PqkOrrI6Vv1pRENmGG5KX4nA6mb1RJ0ZSqqIrbLH4VkSWue9eGg4sAZYWsM0moImINBQRP2AgrruozhKRyByLPYCd7vVVRMTf/TwcuBbXDH0VVkDz5lTu1YsaSzfT3lGXiTETyXBkWB0rf+2G4ff3DobVP8GsjQfI1ImRlKrQCnuBewzwMdAaaAN8bIy5aGc8Y0wW8AiwDFcRmGtzARsrAAAc/0lEQVSMiRWRcSKSfXfTaBGJFZFfgdHAcPf65kCMe/1KYIIxpkIXC4CIR0cjdjuPbYrg4OmDzNw50+pI+WvVH3yDuC94DUdPpfP9zqMFb6OU8lpSrs+NF0F0dLSJifGSCYUu4tg775D4vw+Z92Q0S4L+YHHvxYQHhlsdK28LHsLsWMhNfERk9Qhm3NvR6kRKqSISkc3u68MXVdB8FqdF5FQej9Micqr04qps1e65F3t4OHd8l0J6Vhr//eW/VkfKX7thSMYZnqu/k7V7/mZvwhmrEymlPKSgIcpDjTGV8niEGmMqlVXIS0l2Rz3nth3868x1zP9jPjsSy+kZuLodIKIZnU4vwccmzNRpV5WqsAp7gVuVobC+ffC7rDEdF/xOuE8Yr298vXzeSisC7Ybhe+QX7m6SzBcxh0jN0ImRlKqItFiUQ9kd9bIOHuL5+CvZcmwLyw4sszpW3toMBLsfIwJ/5FRaFl9vO1zwNkopr6PFopwK/sc/CL7mamp/8RNt/C/jzZg3Scsqh53fgqpC8x7U3L+AlhG+OnS5UhWUFotySkSoPmYMzlOnGLOzIfHJ8UyNnWp1rLy1H4akneTpBr+zLe4kvx4q54MhKqWKTItFOZbdUc//qxX0Cb6WydsncyT5iNWxLtSgE1RtxNVJiwnys+vRhVIVkBaLci7isUfBbmfIasHhdPD2lretjnQhEWg3FJ9D67m3uYNFvx4mKaUc9z5XShWZFotyzrdGDardPYKs5asY7XcLS/YuYeuxrXm2HfHtCEZ8O6KME7q1uRNsPgz1X016lpN5m3ViJKUqEi0WXqDq3fdgDw+n04K9RASE8/rG13GacjYWU2gNaNqV8D1f0rFeCDN/PojTWQ5v91VKFYsWCy+Q3VEv/ZdfeT69C9sTt7N472KrY12o/XBI+Zt/1dvDvr+T+enPv61OpJQqJVosvERY3z74N7mMejNW0SasJW9vfpuUzBSrY52v8T+hcl2ij39N1WA/vdCtVAWixcJLiI8P1ceMIfPgIcb+1YaE1AQ+/e3TgjcsSzY7tB2Cbe8q7mtlZ/mOo8SfTLU6lVKqFGix8CLBnToRfM3V+E9bSJ8aNzMtdhpxp8vZheS2Q0CEO31XY4DPdWIkpSoELRZeRESo/tRTOE6dYnhMCHabnTc3v2l1rPNVrgOX3UTlXXP4Z5OqzN54UCdGUqoC0GLhZQKaNaNy796kz5nPwxH9WH5gOZuObLI61vnaDYPT8Yyut59jp9NZvkMnRlLK22mx8EIRj44Gu50blv5FreBaTNg4AYezHI322vQWCK5O62MLqB0WyGfr9UK3Ut5Oi4UXyu6ol/ztdzwd3IffT/zOl398aXWsc+y+0HYw8sd33N82gPV7E9lz7LTVqZRSJaDFwktld9RrOH017au3471f3iPLmWV1rHPaDQXjpJ9tNb52YcYGnRhJKW+mxcJL2UOCiRg9itRffmFs8vUkpScRnxxvdaxzqjaChv8gOHYWt19Rgy83x5GSUY6KmVKqSLRYeLGwPq6Oer4fz6Ffw54cSzlWvua8aDcMkg7yYP04TqdnsWirToyklLfSYuHFznXUO8iI32thExsHTx8sP1OwNu8OgVVpGvcVzWqGMn39gfKTTSlVJFosvJyro941pHwyjV4b4VTGKYZ/O5zdx3dbHQ18/KHNIGTXEu5pF8qO+FP8ohMjKeWVtFh4OVdHPdeMel03OekX48u+k/u4Y/EdvLLhFZLSLP5ybj8MnJn0YDXBOjGSUl5Li0UFkN1Rr9LJTK7dbePr3l8z8PKBzPt9Ht0WdGPOrjnW9cOIuBzqXoX/rzPo07Y2i7fFcyJZJ0byVh2n9KXjlL5WxyixirIfUHb7osWigoh4dDRGICwxncr+lXmm4zPM7T6XplWa8srPrzBg8QA2H91sTbh2QyHxD+6rf4SMLCdfbL60xouqSF9M6tKlxaKC8K1Rg9NhfgQnZxH/0ks4k5NpWqUpk26exMTrJ3Iy4yTDvx3OU2ue4mhyGQ+/0bIX+Fei3v4v6NCgKjM26MRISnkbLRYVyMkqfpyq7EvSnLns7dmLlE2bEBFuaXALC3su5IHWD/D9ge/pvqA7n/72KRmOMjod5BcMrfrDjoUMb1eZg8dTWPNHwkU3GfDRegZ8tL5s8imlCqTFoiIRISk8gPozPgMRDgwdxtHXJuBMSyPIN4hH2j7Cgl4LuDryat7Z8g69FvZi9aHVZZOt/TDISqOLYw3hIX4F9uje7zeR/X4TyyabUqpAWiwqoKD27Wm0YD5VBg3k+LRp7Ovdh9Rt2wCoG1qXd/75Dh/d9BF2sfPID4/w0IqH2H9yv2dDRbaByCh8t37GgOg6/LDrKH8l6cRISnkLjxYLEekqIrtFZI+IPJ3H68NFJEFEtrof9+Z4bZiI/OF+DPNkzorIFhxMzRdeoN7kSTjT0tg/cBDH3nobZ4br1NM1ta/hqx5f8WT0k2w5toXei3rz1ua3SM5M9lyo9sPg6HaG1j/umhjpZx0vSilv4bFiISJ24H3gVqAFMEhEWuTRdI4xJsr9+NS9bVXgRaAj0AF4UUSqeCprRTF7VEtmj2p53rrga66h0aKFVO7Vi8SPPmJ//ztI27ULAF+7L8NaDmNx78Xc3vB2Jm+fTPf53Vm8d7Fnelpf0Q98g6jxx2xubFad2ZsOkpGlEyMp5Q08eWTRAdhjjNlrjMkAZgM9C7ntLcByY8xxY8wJYDnQ1UM5Kzx7aCi1Xv03df73AVnHE9nX/w7+/vBDTJZrYL/wwHBeue4VZtw2g+pB1Xnmx2cY9u0wdibuLN0gAZXgij7w25cMbV+Nv89ksCz2SOl+hlLKIzxZLGoDOW+oj3Ovy62viGwTkXkiUreI26oiCL3hBhotWkSlLl1IePsd9g+6k/Q//zz7epuINsy6fRYvX/MyB04dYMDiAYxbP44TaSdKL0S74ZCZzHVpa6hbNZDPtEe3Ul7Bk8VC8liX+9zG10ADY0xrYAUwrQjbIiL3i0iMiMQkJFz8Vkzl4lOlCrXf/D9qv/UmmYcOsa93HxKnTMU4XD28bWKjT5M+fN37awY3H8xXf3xFt/nd+HzX56UzX0adaIhojm3LNIZ0rM/GfcfZfUQnRlKqvPNksYgD6uZYrgOcN0a1MSbRGJPuXvwEaF/Ybd3bf2yMiTbGREdERJRa8EtBpVtvpdHXiwi+7jqOvf46B4YNI+PguQvOlfwqMbbDWOZ1n0fzqs159edXGbB4QMnn+xZxXeg+vIWB9U7i52Nj5s96dKFUeefJYrEJaCIiDUXEDxgILMrZQEQicyz2ALJPki8DbhaRKu4L2ze716lS5BMRQZ333yNywmuk7/6dvb16c+Lzz8+7uH1Zlcv45OZPeLPzm5zOOM3dy+5mzOoxHEkuwbWG1gPA7k/lnZ/TrVUkX235i+R0nRhJqfLMY8XCGJMFPILrS34nMNcYEysi40Skh7vZaBGJFZFfgdHAcPe2x4HxuArOJmCce526iCldpzCl65QibSMihPXqRaNFCwmKiuLIy+M4dM+9ZMbHn9emS/0uLOy1kAfbPMjKQyvpsaAHH2/7mHRH+kXePR9BVaFFD9g2hyHRNTiTnsWCrX8V/X2UUmXGo/0sjDFLjTFNjTGNjTH/dq97wRizyP38GWNMS2NMG2PMDcaYXTm2nWyMucz9KNo3oCoy38hI6k76lJovvUTK1q3s7d6DpK/mn3eUEegTyENRD7Gw10KurXUt//3lv/Ra0IuVB1cW/VbbdsMg7SRtT6+iRWQlPtOJkZQq17QHtzpLRKgycACNFi7Av9nlxD/7LHEPPUxWrpsHaofU5q0b3uKjLh/hZ/dj9MrRPPj9g+w7ua/wH9bgOqjaGNkynSFX1WfXkdNsPlCKd10ppUqVFgt1Ab+6dak/fTrVnx5L8rp17O3WnVPffHNBu2tqXcO8HvMYEz2GX4/9Sp+Fffi/mP/jTMaZgj9ExDV0+cF19Kp7hlB/H50YSalyTIuFypPYbFQbPpyG87/Ct359/nr8Cf564gmyTpz/27+vzZehLYfyde+v6da4G1Njp9J9QXe+/vNrnKaA3tlRd4LNh6DfZtK3fR2W/naExDPFuAailPI4LRbqovwbNaLBrJlEPPYYp5avYG/3Hpz+4YcL2oUHhjP+2vHMum0WkcGRPLv2WYZ+M5TYxNj83zykOlx+K/z6OUOia5DhcDI3Js6De6OUKi4tFqpA4uND+MgHaPjFXHzCw4l76GEOP/MsjtMXdqZrFdGKGbfNYNw14zh0+hCDFg/ipXUvcTwtn5vZ2g2HlEQuO/EjVzWqysyfD+DQiZGUKne0WKhCC2jWjIZz51DtwZGcXLSIvd17cOanny5oZxMbvZv0ZnHvxQxpMYSFexbSbX43Zu6ceWEv8MY3QOW6sHkaQ66qT9yJVNb8rr3xlSpvtFioIhE/P6o/+igNPp+FLSiIQ/fcS/zLL+NMvnBo81C/UJ668inm9ZhHy2otmbBxAv2/7s/G+I3nGtns0PYu2LuSmyPTiQj11/GilCqHtFioYgls3ZqGX31J1REjSJo9h729epMSE5Nn28Zhjfm4y8e83fltUrNSuee7e/jXqn8Rf8bd8a/tEBAbfttmMOjKuqzcfQxHRkgZ7o1SqiBaLFSx2QICqDH2Kep/Nh2AA3cN5ejrb+BMS7ugrYhwY/0bWdBzAQ9FPcTquNX0WNCDD3/9kPSQcLisC2ydycDoWgiQcvLyMt4bpdTFaLFQJRYUHU2jBfMJGziA41OmsK9P37PTuOYW4BPAg20eZFGvRXSq04n3t75PzwU9+b5hNOZ0PLWO/chNzWuQerIpxqn/PJUqL/T/RlUqbMHBRL74InUnfYozJYX9g+7k2NtvY9zTuOZWK6QWb3Z+k09u/oQAewCP/fEZI2vXYe/mjxlyVX2MI5C0Mw3KdieUUvnysTqAqlhCrr2WRosWcvS1CSR++BFnVq2m1oTXCGjWLM/2V0VexRc9vmDOrjl8EPMmfTP3cOex97D7NeD00avp8d5aAnztBGY//Oznlv1sBPq6l/3OtQnwO7997jZ2W17TpSilLkaLhSp19kqVqPXaq4R26UL8Cy+wr/8dRDz8MNXuvQfxufCfnK/NlyEthnBrlRa8+2VfPvvzK0Ia2jGnW5AR3JRkB2RlQlYaZDrcjyxDRpbBIBgjgA2M4Jo3K+dzASMYBIwNEHxsdvzsdgJ8fPD39cHPbsffx06Arw/+Pj6u5z6+BPi62gT4+BLg50Ogjw+Bvj4E+PoQ6OtLkK8PgX4+BPqdex7s50ugry8BPnZ87D7YsGHc83ZlOjPBgMn+z5z7CeA0zvNeA861ydXeaZw4jcHpNDiMweBadjiN66dx4HQ/d55tD06n6zMcxonDGNeyyd7G9fO89zcGh9N5dn16uh8AX/y2xpUvx99j9vP8BoTM/nO48OXzV+R+3Vw479nZNufeM7/PzPt909IDAMPkzd/luZ03SUv3R8rg9x+pKCN9RkdHm5h87sZR1sk6cYKj48dzauk3BLRuTa0Jr+HfqFH+G0zrwfaT+7g7FFL1JKlSheJMq0nsA8uLta2IbDbGRBfUTo8slEe5pnF9k9CbbuLIy+PY17sPEY8/RtWhQxFbHtWg/TCumHc3PajB1oBAPrt7Ayb7N2XjPPvckP3bbx4PnDidrp9n27t/W87e1uE89x45Xz/vPdzPHU4n6VlZpGc5SM/KIi0ri4wsB+mOLNIzs0h3OMh0uF7LcDjIcDjIzPE8NmEPIDSt2gARm/tCoWATQbIfuNeLzbUe1x1kNgQRGyK4t81+LbudnHvNvWyT7Pd2bed6D/cyrk6T9vNet2HL+R6S4z2w5Xg/4dNf5gDwQPtBZ//Kcv5SK9lL+fymm/265PGr8Hnvc8HLcu69c7d3N87vl+vz38u18PbPkwF44qp78tnKe7y5YRI+ZXBkocVClYlKt91G0JVXEv/Cixyb8DpnVnxP5Guv4le37vkNm3WDwKrcmHKabQFBBPoEWhO4FHWc0heABYNfsjZIKZixyzW1zEMdb7M4Scn879cPABja9p8WJym597e+Xyafowf6qsz4RERQ54P3iXztNdJ27WJvz16cmD37/PPNPv4QdScdU1K4f36qdWGVUufRYqHKlIgQ1rsXjb5e5JrG9aWXOXTvfedN40q7odhsULt6inVBlVLn0QvcyjLGGJLmzOHoG/9B7HZqPPcslXv2REQ4/ngEVStnQFA4VKp1/iM0+3ltqBQJ/qFW78pFLb2pBQC3rdhhcZKSqyj7UlH2A0q+L3qBW5V7rmlcBxJ8zTUcfvZZ4p9+htPLVxD58kv8tieMyPBUmvbpDqcOw6m/IG4TpCRe+Eb+ldxFJNJdQGq5isjZ57UhsEpeV02VUoWkxUJZzq9ePepPm8bx6Z+R8NZb7O3WHWPzYU9qJZp2f/v8xplpcPownIo/V0ROx7t+njoMf+6GM0cg9yx9PgE5iknkuSKSs8CEVHeNgquUuoAWC1UuiN1OtRHDCflHJw4//Qw1fksi7Yzh0IMPYQsMxBYchAQGYgsMwhYU5FoXFIgtqCkS2AZb1WDXcmAgNn9/xJzGlpWEpB5DzhYTd4E5tNFVYBwZuUNAaM08jlJyngKLdF2EV+oSo8VClSv+jRvT4PNZrLumFUFpkHnkCCYlBWdKCs7UVJypqeBwFP4N7XZXAQkMxBYUhAQFYQtshS2oIzY/Oza7E/FxYpN0bKRhS0rB5jyDOH7DlrkGG6nYfAw2Hyc2H4P4GGyhVbFVjUTC6uR/lOKvQ6yrikWLhSp3xMeHk5WEk5Wg7fyvznvNGIPJyMCZkoJJTT1XRFJScaYku9alpuJMzi4uKefaJp8rOI5Tp8hKTXFv51pvLhhaPcj9yEsiYk/E5rMVsTvcBSXHw88HCQrEFhxCU1sqBkgY3TPnXp73o/DLua67XGRZkAK2z/E8r/W2C9s1kjQM8PezQ/FmDcX1d+3t+wGufckqg/HOtFgoryIiiL8/Nn9/qFKlVN/bOBw4U9MwqdkFyF1Mchacs8Ulx7ozZ3CeOo7zzEmcyafJTE7GJKfiTMzAmZGEM8MfjPD3wd9LNa81XKfgEg5tsjhHSVWU/QDwJzAs0+OfosVClUvpUvY9t8Vuxx4SDCHBpfq+Rb21Mc/b2fO7xb0o63OuOzsSn8lzvcGZaxvXz+96dADg5gUb8v5cL/Fdr6sA798PyN4XXy7z8OdosVCqnMlr3KSyvu03v09zul+xhVQuuzAeUFH2A87ti6dpD26llFIF0iMLVS5NuLMBAL2tjaGUctMjC6WUUgXSYqGUUqpAHj0NJSJdgXcAO/CpMWZCPu36AV8AVxpjYkSkAbAT2O1ussEYM9KTWZXylJcHNwfAu2eAcKko+1JR9gPKbl88VixExA68D3QB4oBNIrLIGLMjV7tQYDTwc663+NMYE+WpfEoppQrPk6ehOgB7jDF7jTEZwGygZx7txgNvALm7zyqllConPFksagOHcizHudedJSJtgbrGmMV5bN9QRH4RkdUi0imvDxCR+0UkRkRiEhISSi24Ukqp83myWOTVU+RsV1ERsQFvAf/Ko108UM8Y0xZ4ApglIpUueDNjPjbGRBtjoiMiIkoptlJKqdw8eYE7DqibY7kOcDjHcihwBbDK3WO1JrBIRHoYY2KAdABjzGYR+RNoCuhUeJeIFpEX/G6glLKQJ48sNgFNRKShiPgBA4FF2S8aY04aY8KNMQ2MMQ2ADUAP991QEe4L5IhII6AJsNeDWZVSSl2Ex44sjDFZIvIIsAzXrbOTjTGxIjIOiDHGLLrI5v8AxolIFuAARhpjjnsqqyp/pnSdYnUEpVQOHu1nYYxZCizNte6FfNp2zvH8S+BLT2ZTSilVeNqDWymlVIG0WCillCqQ5DnRiheKjo42MTF6s5RSShWFiGw2xkQX1E6PLJRSShVIi4VSSqkCabFQSilVIC0WSimlCqTFQimlVIG0WCillCqQFgullFIF0mKhlFKqQFoslFJKFajC9OAWkQTgQAneIhz4u5TiWKmi7AfovpRXFWVfKsp+QMn2pb4xpsDZ4ypMsSgpEYkpTJf38q6i7AfovpRXFWVfKsp+QNnsi56GUkopVSAtFkoppQqkxeKcj60OUEoqyn6A7kt5VVH2paLsB5TBvug1C6WUUgXSIwullFIF0mLhJiLjRWSbiGwVke9EpJbVmYpLRP4jIrvc+zNfRMKszlRcItJfRGJFxCkiXnfnioh0FZHdIrJHRJ62Ok9JiMhkETkmItutzlISIlJXRFaKyE73v61Hrc5UXCISICIbReRX97687LHP0tNQLiJSyRhzyv18NNDCGDPS4ljFIiI3Az8YY7JE5HUAY8xYi2MVi4g0B5zAR8CTxhivmQ5RROzA70AXIA7YBAwyxuywNFgxicg/gDPAdGPMFVbnKS4RiQQijTFbRCQU2Az08sa/FxERINgYc0ZEfIG1wKPGmA2l/Vl6ZOGWXSjcggGvraLGmO+MMVnuxQ1AHSvzlIQxZqcxZrfVOYqpA7DHGLPXGJMBzAZ6Wpyp2Iwxa4DjVucoKWNMvDFmi/v5aWAnUNvaVMVjXM64F33dD498d2mxyEFE/i0ih4DBwAtW5ykldwPfWB3iElUbOJRjOQ4v/VKqqESkAdAW+NnaJMUnInYR2QocA5YbYzyyL5dUsRCRFSKyPY9HTwBjzHPGmLrATOARa9NeXEH74m7zHJCFa3/KrcLsi5eSPNZ57RFrRSMiIcCXwGO5zix4FWOMwxgThesMQgcR8cgpQh9PvGl5ZYy5qZBNZwFLgBc9GKdECtoXERkGdANuNOX8wlQR/l68TRxQN8dyHeCwRVlUDu7z+18CM40xX1mdpzQYY5JEZBXQFSj1mxAuqSOLixGRJjkWewC7rMpSUiLSFRgL9DDGpFid5xK2CWgiIg1FxA8YCCyyONMlz31ReBKw0xjzptV5SkJEIrLvdhSRQOAmPPTdpXdDuYnIl8DluO68OQCMNMb8ZW2q4hGRPYA/kOhetcGL7+zqDfwXiACSgK3GmFusTVV4InIb8DZgByYbY/5tcaRiE5HPgc64Rjg9CrxojJlkaahiEJHrgB+B33D9/w7wrDFmqXWpikdEWgPTcP37sgFzjTHjPPJZWiyUUkoVRE9DKaWUKpAWC6WUUgXSYqGUUqpAWiyUUkoVSIuFUkqpAmmxUKoIRORMwa0uuv08EWnkfh4iIh+JyJ/uEUPXiEhHEfFzP7+kOs2q8k2LhVJlRERaAnZjzF73qk9xDczXxBjTEhgOhLsHHfweGGBJUKXyoMVCqWIQl/+4x7D6TUQGuNfbROQD95HCYhFZKiL93JsNBha62zUGOgLPG2OcAO7RaZe42y5wt1eqXNDDXKWKpw8QBbTB1aN5k4isAa4FGgCtgOq4hr+e7N7mWuBz9/OWuHqjO/J5/+3AlR5JrlQx6JGFUsVzHfC5e8TPo8BqXF/u1wFfGGOcxpgjwMoc20QCCYV5c3cRyXBPzqOU5bRYKFU8eQ0/frH1AKlAgPt5LNBGRC72/6A/kFaMbEqVOi0WShXPGmCAe+KZCOAfwEZc01r2dV+7qIFr4L1sO4HLAIwxfwIxwMvuUVARkSbZc3iISDUgwRiTWVY7pNTFaLFQqnjmA9uAX4EfgKfcp52+xDWPxXZc84b/DJx0b7OE84vHvUBNYI+I/AZ8wrn5Lm4AvG4UVFVx6aizSpUyEQkxxpxxHx1sBK41xhxxzzew0r2c34Xt7Pf4CnjGi+cfVxWM3g2lVOlb7J6Qxg8Y7z7iwBiTKiIv4pqH+2B+G7snSlqghUKVJ3pkoZRSqkB6zUIppVSBtFgopZQqkBYLpZRSBdJioZRSqkBaLJRSShVIi4VSSqkC/X9ggbY+0ZCHAAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# CV误差曲线\n",
    "test_means = -grid.cv_results_['mean_test_score']\n",
    "test_stds = grid.cv_results_['std_test_score']\n",
    "train_means = -grid.cv_results_['mean_train_score']\n",
    "train_stds = grid.cv_results_['std_train_score']\n",
    "\n",
    "n_Cs = len(Cs)\n",
    "number_penaltys = len(penaltys)\n",
    "test_scores = np.array(test_means).reshape(n_Cs, number_penaltys)\n",
    "train_scores = np.array(train_means).reshape(n_Cs, number_penaltys)\n",
    "test_stds = np.array(test_stds).reshape(n_Cs, number_penaltys)\n",
    "train_stds = np.array(train_stds).reshape(n_Cs, number_penaltys)\n",
    "\n",
    "x_axis = np.log10(Cs)\n",
    "for i, value in enumerate(penaltys):\n",
    "    pyplot.errorbar(x_axis, test_scores[:,i], yerr=test_stds[:,i],label=penaltys[i]+' test')\n",
    "    pyplot.errorbar(x_axis, train_scores[:,i], yerr=train_stds[:,i], label=penaltys[i]+' train')\n",
    "    \n",
    "pyplot.legend()\n",
    "pyplot.xlabel( 'log(C)' )\n",
    "pyplot.ylabel( 'logloss' )\n",
    "pyplot.savefig('LogisticGridSearchCV_c.png' )\n",
    "\n",
    "pyplot.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "上图给出了L1正则和L2正则下，不同正则参数（C)对赢得模型在训练集上和测试集上的score（正确率）\n",
    "训练集上C越大的模型性能越好；在测试集上当C=1时性能最好"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 正确率做评价指标"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "d:\\python\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "d:\\python\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "d:\\python\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "d:\\python\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "d:\\python\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "d:\\python\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "d:\\python\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "d:\\python\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "d:\\python\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "d:\\python\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "d:\\python\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "d:\\python\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "d:\\python\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "d:\\python\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "d:\\python\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "d:\\python\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "d:\\python\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "d:\\python\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "d:\\python\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "d:\\python\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "d:\\python\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "d:\\python\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "d:\\python\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "d:\\python\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "d:\\python\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "d:\\python\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "d:\\python\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "d:\\python\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "d:\\python\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "d:\\python\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "d:\\python\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "d:\\python\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "d:\\python\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "d:\\python\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "d:\\python\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "d:\\python\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "d:\\python\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "d:\\python\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "d:\\python\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "d:\\python\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "d:\\python\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "d:\\python\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "d:\\python\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "d:\\python\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "d:\\python\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "d:\\python\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "d:\\python\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "d:\\python\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "d:\\python\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "d:\\python\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "d:\\python\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "d:\\python\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "d:\\python\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "d:\\python\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "d:\\python\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "d:\\python\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "d:\\python\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "d:\\python\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "d:\\python\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "d:\\python\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "d:\\python\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "d:\\python\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "d:\\python\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.7747395833333334\n",
      "{'C': 0.1, 'penalty': 'l2'}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "d:\\python\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "d:\\python\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "d:\\python\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "d:\\python\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "d:\\python\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "d:\\python\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "d:\\python\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "d:\\python\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    }
   ],
   "source": [
    "penaltys = ['l1', 'l2']\n",
    "Cs = [0.001, 0.01, 0.1, 1, 10, 100, 1000]\n",
    "tuned_parameters = dict(penalty=penaltys, C=Cs)\n",
    "\n",
    "lr_penalty = LogisticRegression()\n",
    "\n",
    "# score参数默认是正确率，可以不写\n",
    "grid = GridSearchCV(lr_penalty, tuned_parameters, cv=5)\n",
    "grid.fit(x_train, y_train)\n",
    "\n",
    "print(grid.best_score_)\n",
    "print(grid.best_params_)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>columns</th>\n",
       "      <th>coefficent</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Plasma_glucose_concentration</td>\n",
       "      <td>[0.9977309169127752]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>BMI</td>\n",
       "      <td>[0.5460044910698727]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>pregnants</td>\n",
       "      <td>[0.3616713513905089]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Diabetes_pedigree_function</td>\n",
       "      <td>[0.2599038326076921]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Age</td>\n",
       "      <td>[0.15544357341149667]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Triceps_skin_fold_thickness</td>\n",
       "      <td>[0.05966520979020305]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>serum_insulin</td>\n",
       "      <td>[-0.045135531037685375]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>blood_pressure</td>\n",
       "      <td>[-0.06486609661370457]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                        columns               coefficent\n",
       "1  Plasma_glucose_concentration     [0.9977309169127752]\n",
       "5                           BMI     [0.5460044910698727]\n",
       "0                     pregnants     [0.3616713513905089]\n",
       "6    Diabetes_pedigree_function     [0.2599038326076921]\n",
       "7                           Age    [0.15544357341149667]\n",
       "3   Triceps_skin_fold_thickness    [0.05966520979020305]\n",
       "4                 serum_insulin  [-0.045135531037685375]\n",
       "2                blood_pressure   [-0.06486609661370457]"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.DataFrame({'columns':list(feature_names), 'coefficent':list(grid.best_estimator_.coef_.T)}) # coef是权重系数W的属性\n",
    "df.sort_values(by=['coefficent'],ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "d:\\python\\lib\\site-packages\\sklearn\\utils\\deprecation.py:125: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "d:\\python\\lib\\site-packages\\sklearn\\utils\\deprecation.py:125: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEKCAYAAADjDHn2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3Xl8FfW9//HXJyEh7IGwyE6AgAICagQVl7qg1AWLS4tVW21v1V+rt9dbd3tdsNqqbb3t1Xtb21qrbd3ABYWKWveqCCpbAiiLQCBC2Albts/vjznRQ0jIIclkTpL38/E4j8zM+c6cz0Q8n3w/M/P9mrsjIiJyIClRByAiIslPyUJERGqlZCEiIrVSshARkVopWYiISK2ULEREpFahJgszm2BmS81smZndVM37/czsDTP7xMwWmNmZce/dHNtvqZmdEWacIiJyYBbWcxZmlgp8CowHCoA5wEXunh/X5mHgE3f/PzMbBsx09wGx5SeAMUAv4DVgiLuXhxKsiIgcUJg9izHAMndf4e4lwJPAuVXaONAxttwJWBdbPhd40t33uvtKYFnseCIiEoFWIR67N7Ambr0AGFulzR3AK2Z2DdAOOC1u3w+q7Nv7QB/WtWtXHzBgQD3CFRFpeT766KON7t6ttnZhJgurZlvVmtdFwKPu/iszOxZ43MxGJLgvZnYFcAVAv379mDt3bj1DFhFpWcxsVSLtwixDFQB949b78FWZqdL3gacB3P19IAPomuC+uPvD7p7r7rndutWaGEVEpI7CTBZzgBwzyzazdGAyML1Km9XAqQBmdhhBsiiKtZtsZq3NLBvIAT4MMVYRETmA0MpQ7l5mZlcDs4BU4BF3zzOzKcBcd58O/AT4g5ldS1BmusyD27PyzOxpIB8oA36kO6FERKIT2q2zjS03N9d1zUJEqiotLaWgoIA9e/ZEHUqkMjIy6NOnD2lpaftsN7OP3D23tv3DvMAtIhK5goICOnTowIABAzCr7t6Z5s/d2bRpEwUFBWRnZ9fpGBruQ0SatT179pCVldViEwWAmZGVlVWv3pWShYg0ey05UVSq7+9AyUJEpIpv/f59vvX796MOI6koWTQnfz4reDUHzelcpMVr3779l8sTJkwgMzOTs88+u8b2jz76KOvW7fdoWULefPNN3nvvvTrteyBKFiIijej666/n8ccfP2AbJQsRkRbu1FNPpUOHDjW+P3XqVObOncvFF1/M6NGj2b17Nx999BEnnXQSRx11FGeccQaFhYUA/Pa3v2XYsGGMHDmSyZMn8/nnn/O73/2OBx54gNGjR/POO+80WNy6dVZEWow7X8wjf932WtvlFwZtErluMaxXR24/Z3i9Y6t0wQUX8OCDD/LLX/6S3NxcSktLueaaa3jhhRfo1q0bTz31FLfeeiuPPPIIv/jFL1i5ciWtW7dm69atZGZmctVVV9G+fXuuu+66BosJlCxEwld57eXyGdHG0QDy7jkegOG3vBtxJPVTUbILgJT0thFHUrulS5eyaNEixo8fD0B5eTk9e/YECHoU50/knAmn8s3LfhhqHEoWIiHLK9wGQMP97Sl1VdkD2F24BIA2PQ+ttl1lj+KpK49tnMAOwN0ZPnw477+/fy9nxowZvPrcX5nxyuvce9RR5OXlhRaHrlmIiCSZDh06sGPHDgCGDh1KUVHRl8mitLSUvLw8KioqWLNmDSeNG8vdP72OrVu3UlxcvM++DUnJQkSkEZ1wwglceOGF/POf/6RPnz7MmjVrvzaXXXYZV111FaNHj6a8vJypU6dy4403MmrUKEaPHs17771HeXk5l1xyCUefMpFjTz+Pa6+9lszMTM455xyee+45XeAWEWlqiouLv1xO5Av8/PPP5/zzz/9yffTo0bz99tv7tXv33Xf3K6kNGTKEBQsW1Dfk/ShZiIhUkQzXKpKNylAiIlIrJQsREamVkoWIiNRKyUJERGqlZCEiUpVGPd5PqMnCzCaY2VIzW2ZmN1Xz/gNmNi/2+tTMtsa9d5+Z5ZnZYjP7rWn2EhFpoiqHKJ83bx7HHnssw4cPZ+TIkTz11FPVtk/GUWdDu3XWzFKBh4DxQAEwx8ymu3t+ZRt3vzau/TXAEbHl44BxwMjY2+8CJwFvhhWviEjY2rZty2OPPUZOTg7r1q37chTZzMzMfdo9+uijjBgxgl69eh30Z7z55pu0b9+e4447rqHCBsLtWYwBlrn7CncvAZ4Ezj1A+4uAJ2LLDmQA6UBrIA1YH2KsIiKhGzJkCDk5OQD06tWL7t27U1RUtE+bgx2i/MiTzmbMqec26SHKewNr4tYLgLHVNTSz/kA28DqAu79vZm8AhYABD7r74mr2uwK4AqBfv34NGryINEP/uAm+WEh6bNRZahp19ovYE9CJXLc45HD4+i8OOpQPP/yQkpISBg0atM/2gx2iPP+9l2ndOp29bQ5pskOUV3eNwWtoOxmY6u7lAGY2GDgM6BN7/1UzO9Hd93ne3d0fBh4GyM3NrenYIiJJpbCwkEsvvZS//OUvpKQcuMBT2xDll199fZMforwA6Bu33geo6YrNZOBHceuTgA/cvRjAzP4BHAPsPziKiEiiYj2AklqGKA9zDpLt27dz1lln8bOf/Yxjjjmm1vYtYYjyOUCOmWWbWTpBQphetZGZDQU6A/G/idXASWbWyszSCC5u71eGkuYrr3Dbl/NAiDQXJSUlTJo0ie985ztceOGFNbZrUUOUu3sZcDUwi+CL/ml3zzOzKWY2Ma7pRcCT7h5fRpoKLAcWAvOB+e7+Ylixiog0hqeffpq3336bRx99lNGjRzN69GjmzZu3X7sWN0S5u88EZlbZdluV9Tuq2a8cuDLM2EREGkvlEOWXXHIJl1xySa3tNUS5SILMK0ilHCrKISU16nCkpWkG86U3NCULSR4V5fD5O7DwGYaWLiaVCrirK7TvAR16QsdesZ89oUOvfX+27hB19CLNmpKFRMsd1n0MC6fComlQvB7SO7A9pRN7rA09x10M2wthxzrYtDxIJnuqufCd3iGWPOKTSpXk0r67eiktlLvT0kcM2vey8MFTspBobPwMFj4TvDavgNR0GHIGHH4h5JzOuvuDe8p7nvLT/fct2QU7CmH7uup/rnwHir+AirJ997PUoJdSY1KJ/WzdvhF+AdJYMjIy2LRpE1lZWS02Ybg7mzZtIiMjo87HULKQxrN9XdB7WPgMFM4HS4HsE+GEn8ChZ0ObzNqPAcFTt1mDgldNKipgZ1HQI6nsmWwv/CqpbFoWJJW91fRSWnesptxVJam061ZrL2V3STkfr97CjD3Hs5t0es9aSkZaChlpqbROSyWjVbAcvGLLrVLj2ny1LS3VWuwXXX316dOHgoKCfYbVKN32BQBpW5v+s7yJnktGRgZ9+vQ5YJsDUbKQcO3aDIunB2Wmz98FHHofBRN+AcMnQYdDwvnclBTo0CN49Tqi5nZ7i2HHF9UkldjPjW8F7weDC8Qdv1XctZQgqext24MVezsyf2sb3t2QzttftGJ7eWtSGEc6ZZS8uYyKOn43pRhfJZZWcQknLWWfBFOZeFq3qpqEqrxfJTFl7HOsVFq3SiElpXkkp7S0NLKzs/fZlnfPDwA47JZ3owipQTXWuShZSMMr2QWf/iNIEJ+9ChWlkJUDJ98CI84/cI+gsbVuD60HQ9fBNbepKIfiDfv1Tkq2rqW4aDUVKxfQdu9rtPXdHEYwTs1kgDQobduRstIS3Iw2vYbhHpQEKtyp+HKZatf3Xa5sE1sucypKq+xfAU7cPhVe4/g68fbGXlX7WGaQYkGPJiW2nFK6B8P59GdH1+33nSRSS3YDNPnzgOBcNpJgr7welCwg1Ef7W4zyUljxZlBiWjIDSoqDEs4xVwXXIQ4ZGXz7NEUpqdCxJ1tbZfHh1t7M3rCZ2Ss3kb9uOxUO6akpjOrbieP7ZTCuewkjOuwkY3eQXNK2F7J77lMYFVibzl8OmNZYl9krk0y5OxUVXyWS8oqvElZFRez9WMKp7v0Kd8oqoLR0Cw60atWpkc4gHGUlQRpt6ucBwbmUWfhf5UoWUncVFVDwYZAg8p6DXZsgIzPoPRx+IfQfF5SDmqhNxXv5cOVmZq/czAcrNrF0/Q7coXWrFI7ol8k1p+QwdmAXjuzXmYy0mr/+18z/AIDhl0xrrNC/ZASJqaGSU949xwMw/KbXGuiI0Wgu5wHBuXSjrPaG9aRkIQdvfV7sTqZpsG01tGoDQ78eJIjBp0Kr1lFHWCcbduxh9oqg1zB7xWY+2xA8ddsmLZWj+nfmrMN7MnZgFqP6dqJ1K92CKy2LkoUkZssqWDQ1uA6xIT+4DXXQKXDKT+HQM5vkQ3GF23bvkxxWbNwJQLv0VHIHdGHSkb0Zm53F4b07kd6q6faQRBqCkoXUrLgI8p8PehFrZgfb+h4DZ/4yuJOpXddo4ztIazbvYvbKzcxesYnZKzezenMwAU6HjFaMGdCFyWP6MjY7i+G9OtIqVclBJJ6Shexr747gAvXCZ2D5G8Eto92Hw6m3B9ciOvePOsKEuDurNu36stcwe+Vm1m4N7oDJbJvGmAFd+O5xAxib3YXDenYktZncJioSFiULgbK9sOy1IEEsfRnKdkOnfjDux3D4BdBjeNQR1srdWV60My45bGL99r0AZLVLZ+zALlxx4kDGDuzCkO4dms0zBCKNRcmipaooh1X/ChJE/gvBeEtts+CIS4IL1X3HJPWtrhUVzmcbivfpOWwsDpJD9w6tGTswi7HZXThmYBcGdWuvp59F6knJoiVxD4bZWPgMLHo2eMgsvX0w1MbhF8LAkyA1Leooq1VR4Sz+YvuXvYYPV25my65SAHp1yuCEnK6Mze7C2IFZDMhqq+Qg0sCULFqCTcuDu5gWPgObPoOUNMg5HQ6/G4ZMCMZaSjIVDssrDuG9t1d8mRy27wnuJe/bpQ2nHtYj1nPIok/nNkoOIiFTsmiudnwR9B4WPhMMAY7BgOPhuGtg2ERo0znqCA/oV7sn8mbZcJi5mOyu7Tjz8J6MHdiFsdlZ9MpsE3V4Ii1OqMnCzCYAvyF4gPSP7v6LKu8/AJwcW20LdHf3zNh7/YA/An0BB85098/DjLfJqyiDXRvhLxNh5duAQ8/RcPrdMOK8YLTUJmDGgkLeLBvOpPTZ3HTdLfToWPdhlUWkYYSWLMwsFXgIGA8UAHPMbLq751e2cfdr49pfA8QPD/oYcLe7v2pm7YGKsGJt0txh9fvw8ePB0BteEbxOujG4k6lrTtQRHpRNxXu57YVF5KQUcnnrN+jRcUrUIYkI4fYsxgDL3H0FgJk9CZwL5NfQ/iLg9ljbYUArd38VwN2LQ4yzadqxHub/HT75azA3Q3oHaNc9GDb7yreS+k6mA7njxXy27yllSpuXSLWmP9eASHMRZrLoDayJWy8AxlbX0Mz6A9nA67FNQ4CtZvZsbPtrwE3uVScVaGHKy+CzV+CTx+HTWcEDc/2OheP/E4Z/A/72zaBdE00ULy/6ghfnr+Mn44cwYPbGqMMRkThhJovqvrFq+lNxMjA1Lhm0Ak4gKEutBp4CLgP+tM8HmF0BXAHQr1+/+kecrDYuCxLE/CeCOarbdYfjroYjLm1yZaaabNlZwk+fX8TwXh256muD+HR21BGJSLwwk0UBwcXpSn2AdTW0nQz8qMq+n8SVsJ4HjqFKsnD3h4GHAXJzc5tXzaJkZ/Cw3MePw+r3goH7hpwRPDSXc3rSPg9RV1NeymfrrhIe+94Y0jQuk0jSCTNZzAFyzCwbWEuQEL5dtZGZDQU6A+9X2bezmXVz9yLgFGBuiLEmB3dY+zF88lgw/HfJDugyCE67A0ZdFN4UpBF7LX89z32ylh+fmsOwXh2jDkdEqhFasnD3MjO7GphFcOvsI+6eZ2ZTgLnuPj3W9CLgSXf3uH3Lzew64J8WPG31EfCHsGKN3M5NsOCpoNS0IT+YH2L4N4IyU//jmuw1iERs21XKLc8t5NBDOvCjkw8wtamIRCrU5yzcfSYws8q226qs31HDvq8CI0MLLmoV5bDijaDMtHQmlJdAryPh7AeC0V0zmv50j4m4a0Y+m3aW8MhlR2vOCJEkpie4G9uWVTDvb/DJ32B7QfAkde734chLm8Torg3pjaUbmPpRAVefPJgRvfdNjlOy7geCOxtEJHpKFo2hdA8seSkoM614K9g26GQ4/S449KwmOw1pfWzfU8rN0xYypEd7rjlV5SeRZKdkEaYvFgZlpoVPw+4twRwRX7sJRn8bMpvxrb4JuGfGYjbs2MPvLx2n+axFmgAli4a2e2swV/XHj0PhPEhND4YAP/JSyP4apKgu//anRTw5Zw1XnTSIUX0zow4ndMN7tozrT9K8KVk0BHf4/N2gzJT/ApTtgR4jYMK9MPKb0LZL1BEmjeK9Zdz87EIGdWvHf5zWPB4oFGkJlCzqY3th7GL1X2HLSmjdMSgxHXEp9DqiWd/yWlc/n7mYddt2M/Wq48hIU/lJpKlQsjhY5aXw6ctBmWnZq8EIr/2PD65FHDYxKScSShbvLdvI32av5gcnZHNU/+SeT0NE9qVkkaiiT4Mnq+c/CTuLoP0hMO7HQS8ia1DU0SW9nXvLuGHaArK7tuMnpw+NOpzGdfmMqCNoMMNveTfqEBpEczkPaLxzUbI4kL3FkPdccC1izezY+EwTgovVg8dDqn59ibrv5SWs3bqbp688VuUnkSZI33ZVuUPBHPj4sSBRlBRD1mA47c7Y+Ew9oo6wyflgxSb+8v4qLh83gKMH6GK/SFOkZFGpvBTeezDoRRQtgbS2MHxSUGbqd4wuVtfR7pJybpy2gH5d2nL9GS2s/CTSjChZbC+EosWwazMUzIbeuXDOb2D4eZChEVDr6/5ZS1m1aRdP/OAY2qYn/s/tqSuPDTEqETlYShZtMoO5Izr0hEufhe6HRR1RszH38838+b2VXHpMf44dlBV1OCJSD0oWaW2g11FBmUmJosHsKS3nhqkL6J3Zhpu+fmjU4YhIPSlZgK5HhODXr37Kio07+du/jaVda/0zE2nqNFCRNLiPV2/hj++s4KIx/Rg3uGvU4YhIA1CykAa1p7Sc65+ZzyEdM7jlTJWfRJoL1QekQf3mn5+xvGgnf/neGDpkpEUdjog0EPUspMHMX7OV37+1nG/m9uGkId2iDkdEGlCoycLMJpjZUjNbZmY3VfP+A2Y2L/b61My2Vnm/o5mtNbMHw4xT6m9vWTnXT51P9w4Z3HrWsKjDEZEGFloZysxSgYeA8UABMMfMprt7fmUbd782rv01wBFVDnMX8FZYMUrDefD1ZXy6vpg/X3Y0ndqo/CTS3ITZsxgDLHP3Fe5eAjwJnHuA9hcBT1SumNlRQA/glRBjlAawaO02/vfN5Zx3ZG9OPrR71OGISAjCTBa9gTVx6wWxbfsxs/5ANvB6bD0F+BVw/YE+wMyuMLO5Zja3qKioQYKWg1NSVsH1UxeQ1S6d288eHnU4IhKSMJNFdU+6eQ1tJwNT3b08tv5DYKa7r6mhfXAw94fdPdfdc7t10wXVKPzvm8tYXLiduycdTqe2Kj+JNFdh3jpbAPSNW+8DrKuh7WTgR3HrxwInmNkPgfZAupkVu/t+F8klOosLt/Pg68s4d3Qvxg/T0O0izVmYyWIOkGNm2cBagoTw7aqNzGwo0Bl4v3Kbu18c9/5lQK4SRXIpLa/g+qnzyWybxh3nqPwk0tyFVoZy9zLgamAWsBh42t3zzGyKmU2Ma3oR8KS711SikiT0+7eWs2jtdn72jRF0bpcedTgiErJQn+B295nAzCrbbquyfkctx3gUeLSBQ5N6+HT9Dn77z2WcNbInE0b0jDocEWkEeoJbDkpZeQXXPzOf9hmtmDJR5SeRlkJjQ8lB+cM7K5lfsI0Hv30EWe1bRx2OiDSShHoWZjYi7EAk+S3bUMwDr33KhOGHcNbhKj+JtCSJlqF+Z2YfmtkPzSwz1IgkKZVXONdPnU/b9FTu+sYITBNGibQoCSULdz8euJjguYm5ZvZ3MxsfamSSVB55dyWfrN7KnROH062Dyk8iLU3CF7jd/TPgp8CNwEnAb81siZmdF1ZwkhxWFBXzy1eWctphPZg4qlfU4YhIBBK9ZjHSzB4geF7iFOAcdz8stvxAiPFJxCoqnBunLaB1qxTumaTyk0hLlejdUA8CfwBucffdlRvdfZ2Z/TSUyCQp/OX9z5nz+RZ+deEounfMiDocEYlIrckiNi/FGnd/vLr3a9ouEbh8RoMebtWmndz78hJOHtqN846sdsBgEWkhai1DxUaCzTIzjenQglRUODdMXUBaSgr3nHe4yk8iLVyiZahVwL/MbDqws3Kju/86lKikTr71+2AsxqeuPLbex/rb7FXMXrmZ+84fSc9Obep9PBFp2hJNFutirxSgQ3jhSDJYs3kXP//HEk4c0o0Lc/tEHY6IJIGEkoW73xl2IJIc3J2bnl1Aihk/V/lJRGISShZm1g24ARgOfHlLjLufElJcEpEnPlzDv5Zt4u5JI+idqfKTiAQSfSjvb8ASgnmy7wQ+J5jcSJqRtVt3c8/MxRw3KItvj+kXdTgikkQSTRZZ7v4noNTd33L37wHHhBiXNDJ356ZpC6hw597zR6r8JCL7SPQCd2nsZ6GZnUVwsbv5XPls4OcTmqJn5hbwzmcbmXLucPp2aRt1OCKSZBJNFj8zs07AT4D/AToC14YWlTSqwm27ueulfMZmd+GSsf2jDkdEklCid0O9FFvcBpyc6MHNbALwGyAV+KO7/6LK+w/EHa8t0N3dM81sNPB/BEmpHLjb3Z9K9HMlce7OLc8upKzCue+CkaSkqPwkIvtL9G6oPwNedXvs2kVN+6QCDwHjgQJgjplNd/f8uP2vjWt/DXBEbHUX8B13/8zMegEfmdksd9+aSLySuGc/XssbS4u47exh9M9qF3U4IpKkEi1DvRS3nAFMIrhucSBjgGXuvgLAzJ4EzgXya2h/EXA7gLt/WrkxNljhBqAboGTRgNZv38OdL+Zx9IDOXHbcgKjDEZEklmgZalr8upk9AbxWy269gTVx6wXA2Ooamll/gttyX6/mvTFAOrA8kVglMe7Orc8tYm9ZBfddMErlJxE5oIQnP6oiB6jtRvzqvn32K2XFTAamxgYt/OoAZj2Bx4HL3b1ivw8wu8LM5prZ3KKiogTClkrT56/jtcXrue70oWR3VflJRA4s0cmPdpjZ9soX8CLBjHkHUkAwDWulPtRcupoMPFHlMzsCM4CfuvsH1e3k7g+7e66753br1i2RUxGgaMdebp+exxH9Mvne8dlRhyMiTUCiZai6DB44B8gxs2xgLUFC+HbVRmY2FOgMvB+3LR14DnjM3Z+pw2dLDdyd/3p+EbtKyrn/glGkqvwkIglItGcxKfacReV6ppl940D7uHsZcDUwi2A61qfdPc/MppjZxLimFwFPunt8ieqbwInAZWY2L/YaneA5yQHMWFjIy3lfcO1pQxjcvX3U4YhIE5Ho3VC3u/tzlSvuvtXMbgeeP9BO7j4TmFll221V1u+oZr+/An9NMDZJ0Kbivdz2Qh6j+nTiByeo/CQiiUv0And17RJNNJIkbpueR/GeMu67YBStUut6b4OItESJfmPMNbNfm9kgMxsYe/L6ozADk4b18qJCZiwo5N9PHczQQzR/lYgcnESTxTVACfAU8DSwG/hRWEFJw9qys4SfPr+I4b06cuVJg6IOR0SaoETvhtoJ3BRyLBKSO17MY+uuUh7//ljSVH4SkTpI9G6oV80sM269s5nNCi8saSiv5q/nhXnruPqUwRzWs2PU4YhIE5Xon5ld4wfxc/ctQPdwQpKGsnVXCbc8t5BDD+nAD782OOpwRKQJSzRZVJjZl8N7mNkAah66Q5LElJfy2byzhF9eOIr0Vio/iUjdJXr7663Au2b2Vmz9ROCKcEKShvD6kvU8+/FarjllMCN6d6p9BxGRA0j0AvfLZpZLkCDmAS8Q3BElSWjb7lJueXYRQ3q05+pTVH4SkfpLdPKjfwN+TDAY4DzgGIKxnE4JLzSpq7tn5FNUvJeHv3MUrVulRh2OiDQDiRayfwwcDaxy95MJZrTTmOBJaOuuEp6eW8AVJw5kZJ/M2ncQEUlAoslij7vvATCz1u6+BBgaXlhSF2UVzsqNuxjcvT0/PjUn6nBEpBlJ9AJ3Qew5i+eBV81sC7VPqyqNbM3mXZSUV3DfBSPJSFP5SUQaTqIXuCfFFu8wszeATsDLoUUlB+1fyzayYcdeenbK4Mh+naMOR0SamYMeOdbd36q9lTSmnXvLuHHaAjLSUuiT2SbqcESkGdKTWs3AvS8vYe3W3Qzs2o4UzXwnIiFQsmji3l++icfeX8Xlx2XTISMt6nBEpJlSsmjCdpUE5af+WW25/gzdnCYi4Qk1WZjZBDNbambLzGy/Ic7N7IG4ObY/NbOtce9918w+i72+G2acTdX9s5ayevMu7j1/JG3SdfeTiIQntKlRzSwVeAgYDxQAc8xsurvnV7Zx92vj2l9D8LAfZtYFuB3IJRiw8KPYvlvCirepmfP5Zh5973O+e2x/jhmYFXU4ItLMhdmzGAMsc/cV7l4CPAmce4D2FwFPxJbPAF51982xBPEqMCHEWJuU3SXl3DB1AX06t+GGCYdGHY6ItABhJovewJq49YLYtv2YWX8gG3j9YPdtiX796lJWbtzJveeNpF3r0DqHIiJfCjNZVHcPZ01zYEwGprp7+cHsa2ZXmNlcM5tbVNQyhqr6aNUW/vTuSi4e24/jBneNOhwRaSHCTBYFQN+49T7UPETIZL4qQSW8r7s/7O657p7brVu3eoab/PaUlnPD1Pn07NSGm888LOpwRKQFCTNZzAFyzCzbzNIJEsL0qo3MbCjQmWDI80qzgNNjc313Bk6PbWvR/vu1z1hetJOfn3c47VV+EpFGFNo3jruXmdnVBF/yqcAj7p5nZlOAue5emTguAp50d4/bd7OZ3UWQcACmuPvmsGJtCuav2crDby/nW7l9OXFI8+9FiUhyCfXPU3efCcyssu22Kut31LDvI8AjoQXXhOwtK+e6Z+bTo2MGt56t8pOIND7VMpqA//nnMj7bUMyfLz+ajhq1gApWAAAOTElEQVTSQ0QioOE+ktyitdv4v7eWc/6RfTh5aPeowxGRFkrJIomVlFVw3TPzyWqXzm1nD4s6HBFpwVSGSmIPvbGMJV/s4I/fyaVTW5WfRCQ66lkkqfx123nojWV8Y3QvThvWI+pwRKSFU7JIQqXlQfkps206t58zPOpwRERUhkpGv3tzOfmF2/ndJUfRuV161OGIiKhnkWyWfrGD377+GWeP7MmEEYdEHY6ICKBkkVTKyiu4fup8OmakcedElZ9EJHmoDJVEHn5nBQsKtvHQt48kq33rqMMREfmSehZJYtmGHfz3q5/x9RGHcNbInlGHIyKyDyWLJFBe4Vz3zALatU5lyrkjog5HRGQ/KkMlgT+9u4J5a7bym8mj6dZB5ScRST7qWURsRVExv3rlU8YP68HEUb2iDkdEpFpKFhEqr3BumLqAjLRU7v7GCMyqm01WRCR6KkNF6NH3Pmfuqi38+puj6N4xo97He+rKYxsgKhGR/alnEZHPN+7k/llLOOXQ7kw6onfU4YiIHJCSRQQqKpwbpi0gLTWFeyYdrvKTiCQ9JYsI/HX2Kj5cuZn/OnsYh3Sqf/lJRCRsoSYLM5tgZkvNbJmZ3VRDm2+aWb6Z5ZnZ3+O23xfbttjMfmvN5M/vNZt38Yt/LOHEId248Kg+UYcjIpKQ0C5wm1kq8BAwHigA5pjZdHfPj2uTA9wMjHP3LWbWPbb9OGAcMDLW9F3gJODNsOJtDO7OjdMWkGLGL85T+UlEmo4wexZjgGXuvsLdS4AngXOrtPkB8JC7bwFw9w2x7Q5kAOlAayANWB9irI3i7x+u5r3lm7jlzMPoldkm6nBERBIWZrLoDayJWy+IbYs3BBhiZv8ysw/MbAKAu78PvAEUxl6z3H1x1Q8wsyvMbK6ZzS0qKgrlJBpKwZZd3DNjMccP7spFY/pGHY6IyEEJM1lUV2PxKuutgBzga8BFwB/NLNPMBgOHAX0IEswpZnbifgdzf9jdc909t1u3bg0afENyd25+diEO/FzlJxFpgsJMFgVA/J/QfYB11bR5wd1L3X0lsJQgeUwCPnD3YncvBv4BHBNirKF6eu4a3vlsIzd//VD6dmkbdTgiIgctzGQxB8gxs2wzSwcmA9OrtHkeOBnAzLoSlKVWAKuBk8yslZmlEVzc3q8M1RQUbtvNz15azDEDu3Dx2P5RhyMiUiehJQt3LwOuBmYRfNE/7e55ZjbFzCbGms0CNplZPsE1iuvdfRMwFVgOLATmA/Pd/cWwYg1LZfmprMK57/xRpKSo/CQiTVOoY0O5+0xgZpVtt8UtO/CfsVd8m3LgyjBjawzTPl7Lm0uLuP2cYfTLUvlJRJouPcEdkvXb9zDlxTzGDOjCd48dEHU4IiL1omQRAnfn1ucWsresgnsvGKnyk4g0eUoWIXhh3jpeW7yB688YSnbXdlGHIyJSb0oWDWzDjj3c8WIeR/bL5PJx2VGHIyLSIJQsGpC781/PL2JXSTn3XTCKVJWfRKSZULJoQC8tKGRW3nr+c/wQBndvH3U4IiINRsmigWws3svt0/MY1acT/3a8yk8i0rwoWTSQ21/Io3hPGfdfOIpWqfq1ikjzom+1BvCPhYXMWFjIj0/LYUiPDlGHIyLS4JQs6mnzzhL+64VFjOjdkStOHBh1OCIioQh1uI+W4I7peWzbXcrj3x9LmspPItJM6dutHl7J+4Lp89dx9ck5HNazY9ThiIiERsmijrbuKuHW5xdxWM+O/PDkQVGHIyISKpWh6mjKS/ls2VnCo5cfrfKTiDR7+parg9eXrOfZj9fyw68NYnivTlGHIyISOiWLg7Rtdyk3P7uQoT06cPUpOVGHIyLSKFSGOkh3z8hnY3EJf/hOLumtlGtFpGXQt91BeHPpBp6eW8CVJw5kZJ/MqMMREWk0oSYLM5tgZkvNbJmZ3VRDm2+aWb6Z5ZnZ3+O29zOzV8xscez9AWHGWpsde4Ly0+Du7fn3U1V+EpGWJbQylJmlAg8B44ECYI6ZTXf3/Lg2OcDNwDh332Jm3eMO8Rhwt7u/ambtgYqwYk3EPTOXsH77Hqb9v+PISEuNMhQRkUYXZs9iDLDM3Ve4ewnwJHBulTY/AB5y9y0A7r4BwMyGAa3c/dXY9mJ33xVirAf07mcbeeLD1fzghIEc0a9zVGGIiEQmzGTRG1gTt14Q2xZvCDDEzP5lZh+Y2YS47VvN7Fkz+8TM7o/1VBpd8d4ybpy2gIFd23Ht+CFRhCAiErkwk0V108R5lfVWQA7wNeAi4I9mlhnbfgJwHXA0MBC4bL8PMLvCzOaa2dyioqKGizzOvf9Ywrptu7n/wpEqP4lIixVmsigA+sat9wHWVdPmBXcvdfeVwFKC5FEAfBIrYZUBzwNHVv0Ad3/Y3XPdPbdbt24NfgLvLd/I4x+s4nvjsjmqf5cGP76ISFMRZrKYA+SYWbaZpQOTgelV2jwPnAxgZl0Jyk8rYvt2NrPKDHAKkE8j2lVSxk3TFjIgqy3XnT60MT9aRCTphJYsYj2Cq4FZwGLgaXfPM7MpZjYx1mwWsMnM8oE3gOvdfZO7lxOUoP5pZgsJSlp/CCvW6tz38lJWb97FveePpE26yk8i0rKZe9XLCE1Tbm6uz507t0GO9eHKzXzr4ff57rEDuGPi8AY5pohIMjKzj9w9t7Z2eoK7it0l5dwwdT59OrfhhgkqP4mIgMaG2s+vXlnK55t28fcfjKVtun49IiKgnsU+Plq1mT/9ayUXj+3HcYO6Rh2OiEjSULKI2VNazvVTF9CrUxtuPvOwqMMREUkqqrMA3/r9+6zevIvCbXt4/PtjaN9avxYRkXjqWRAM6VG4bQ+Tj+7LCTkN/3CfiEhT1+KTxd6yclYU7SQ9NYVbzlL5SUSkOi0+WWzYvheA7K5t6ZiRFnE0IiLJqcUX5/t2acvhvTtiVt24hyIiAupZAChRiIjUosX3LACeuvLYqEMQEUlq6lmIiEitlCxERKRWShYiIlIrJQsREamVkoWIiNRKyUJERGqlZCEiIrVSshARkVopWYiISK3M3aOOoUGYWRGwqh6H6ApsbKBwotRczgN0LsmquZxLczkPqN+59Hf3WudmaDbJor7MbK6750YdR301l/MAnUuyai7n0lzOAxrnXFSGEhGRWilZiIhIrZQsvvJw1AE0kOZyHqBzSVbN5Vyay3lAI5yLrlmIiEit1LMQEZFaKVnEmNldZrbAzOaZ2Stm1ivqmOrKzO43syWx83nOzDKjjqmuzOxCM8szswoza3J3rpjZBDNbambLzOymqOOpDzN7xMw2mNmiqGOpDzPra2ZvmNni2L+tH0cdU12ZWYaZfWhm82Pncmdon6UyVMDMOrr79tjyvwPD3P2qiMOqEzM7HXjd3cvM7F4Ad78x4rDqxMwOAyqA3wPXufvciENKmJmlAp8C44ECYA5wkbvnRxpYHZnZiUAx8Ji7j4g6nroys55AT3f/2Mw6AB8B32iK/10smBO6nbsXm1ka8C7wY3f/oKE/Sz2LmMpEEdMOaLJZ1N1fcfey2OoHQJ8o46kPd1/s7kujjqOOxgDL3H2Fu5cATwLnRhxTnbn728DmqOOoL3cvdPePY8s7gMVA72ijqhsPFMdW02KvUL67lCzimNndZrYGuBi4Lep4Gsj3gH9EHUQL1RtYE7deQBP9UmquzGwAcAQwO9pI6s7MUs1sHrABeNXdQzmXFpUszOw1M1tUzetcAHe/1d37An8Dro422gOr7VxibW4FygjOJ2klci5NlFWzrcn2WJsbM2sPTAP+o0ploUlx93J3H01QQRhjZqGUCFuFcdBk5e6nJdj078AM4PYQw6mX2s7FzL4LnA2c6kl+Yeog/rs0NQVA37j1PsC6iGKROLH6/jTgb+7+bNTxNAR332pmbwITgAa/CaFF9SwOxMxy4lYnAkuiiqW+zGwCcCMw0d13RR1PCzYHyDGzbDNLByYD0yOOqcWLXRT+E7DY3X8ddTz1YWbdKu92NLM2wGmE9N2lu6FizGwaMJTgzptVwFXuvjbaqOrGzJYBrYFNsU0fNOE7uyYB/wN0A7YC89z9jGijSpyZnQn8N5AKPOLud0ccUp2Z2RPA1whGOF0P3O7uf4o0qDows+OBd4CFBP+/A9zi7jOji6puzGwk8BeCf18pwNPuPiWUz1KyEBGR2qgMJSIitVKyEBGRWilZiIhIrZQsRESkVkoWIiJSKyULkYNgZsW1tzrg/lPNbGBsub2Z/d7MlsdGDH3bzMaaWXpsuUU9NCvJTclCpJGY2XAg1d1XxDb9kWBgvhx3Hw5cBnSNDTr4T+BbkQQqUg0lC5E6sMD9sTGsFprZt2LbU8zsf2M9hZfMbKaZXRDb7WLghVi7QcBY4KfuXgEQG512Rqzt87H2IklB3VyRujkPGA2MIniieY6ZvQ2MAwYAhwPdCYa/fiS2zzjgidjycIKn0ctrOP4i4OhQIhepA/UsROrmeOCJ2Iif64G3CL7cjweecfcKd/8CeCNun55AUSIHjyWRktjkPCKRU7IQqZvqhh8/0HaA3UBGbDkPGGVmB/p/sDWwpw6xiTQ4JQuRunkb+FZs4pluwInAhwTTWp4fu3bRg2DgvUqLgcEA7r4cmAvcGRsFFTPLqZzDw8yygCJ3L22sExI5ECULkbp5DlgAzAdeB26IlZ2mEcxjsYhg3vDZwLbYPjPYN3n8G3AIsMzMFgJ/4Kv5Lk4GmtwoqNJ8adRZkQZmZu3dvTjWO/gQGOfuX8TmG3gjtl7The3KYzwL3NyE5x+XZkZ3Q4k0vJdiE9KkA3fFehy4+24zu51gHu7VNe0cmyjpeSUKSSbqWYiISK10zUJERGqlZCEiIrVSshARkVopWYiISK2ULEREpFZKFiIiUqv/Dz9ZBqm0sTOhAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# CV误差曲线\n",
    "test_means = grid.cv_results_['mean_test_score']\n",
    "test_stds = grid.cv_results_['std_test_score']\n",
    "train_means = grid.cv_results_['mean_train_score']\n",
    "train_stds = grid.cv_results_['std_train_score']\n",
    "\n",
    "n_Cs = len(Cs)\n",
    "number_penaltys = len(penaltys)\n",
    "test_scores = np.array(test_means).reshape(n_Cs, number_penaltys)\n",
    "#train_scores = np.array(train_means).reshape(n_Cs, number_penaltys)\n",
    "test_stds = np.array(test_stds).reshape(n_Cs, number_penaltys)\n",
    "#train_stds = np.array(train_stds).reshape(n_Cs, number_penaltys)\n",
    "\n",
    "x_axis = np.log10(Cs)\n",
    "for i, value in enumerate(penaltys):\n",
    "    pyplot.errorbar(x_axis, test_scores[:,i], yerr=test_stds[:,i],label=penaltys[i]+' test')\n",
    "    #pyplot.errorbar(x_axis, train_scores[:,i], yerr=train_stds[:,i], label=penaltys[i]+' train')\n",
    "    \n",
    "pyplot.legend()\n",
    "pyplot.xlabel( 'log(C)' )\n",
    "pyplot.ylabel( 'accuary' )\n",
    "pyplot.savefig('LogisticGridSearchCV_c.png' )\n",
    "\n",
    "pyplot.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "accuary值越大越好，与前面的负log损失相反"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "l2\n",
      "0.1\n"
     ]
    }
   ],
   "source": [
    "print(grid.best_params_['penalty'])\n",
    "print(grid.best_params_['C'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "C 取0.1 penalty取l2时最优"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.36167135,  0.99773092, -0.0648661 ,  0.05966521, -0.04513553,\n",
       "         0.54600449,  0.25990383,  0.15544357]])"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grid.best_estimator_.coef_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>columns</th>\n",
       "      <th>coeffient</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Plasma_glucose_concentration</td>\n",
       "      <td>[0.9977309169127752]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>BMI</td>\n",
       "      <td>[0.5460044910698727]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>pregnants</td>\n",
       "      <td>[0.3616713513905089]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Diabetes_pedigree_function</td>\n",
       "      <td>[0.2599038326076921]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Age</td>\n",
       "      <td>[0.15544357341149667]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Triceps_skin_fold_thickness</td>\n",
       "      <td>[0.05966520979020305]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>serum_insulin</td>\n",
       "      <td>[-0.045135531037685375]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>blood_pressure</td>\n",
       "      <td>[-0.06486609661370457]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                        columns                coeffient\n",
       "1  Plasma_glucose_concentration     [0.9977309169127752]\n",
       "5                           BMI     [0.5460044910698727]\n",
       "0                     pregnants     [0.3616713513905089]\n",
       "6    Diabetes_pedigree_function     [0.2599038326076921]\n",
       "7                           Age    [0.15544357341149667]\n",
       "3   Triceps_skin_fold_thickness    [0.05966520979020305]\n",
       "4                 serum_insulin  [-0.045135531037685375]\n",
       "2                blood_pressure   [-0.06486609661370457]"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.DataFrame({'columns':list(feature_names), 'coeffient':list(grid.best_estimator_.coef_.T)})\n",
    "df.sort_values(by=['coeffient'],ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
